#!/bin/bash

# 此脚本需要在Prometheus节点执行，包括检测Thanos Rule进程状态、端口状态

# 系统ID
system_id=$(cat config.cnf | grep SystemId | awk -F "=" '{print $2}')
# 告警存放文件夹
alert_dir=$(cat config.cnf | grep AlertDir | awk -F "=" '{print $2}')
# 告警缓存文件
alarm_file_tmp="thanos_rule_check_alarm.txt"
echo -n "" >${alarm_file_tmp}
# 本机ip
local_ip=$(ifconfig -a | grep inet | grep -v 127.0.0.1 | grep -v inet6 | awk '{print $2}' | tr -d "addr:")
# Thanos Rule端口
thanos_rule_port=$(cat config.cnf | grep ThanosRulePort | awk -F "=" '{print $2}')
execute_cycle=$(cat config.cnf | grep ExecuteCycle | awk -F "=" '{print $2}')

# Thanos Rule进程状态监控
thanos_rule_pro_check() {
    thanos_rule_pro=$(ps -elf | grep "thanos rule"| grep -v grep | wc -l)
	if [[ ${thanos_rule_pro} == 0 ]]; then
        # 进程状态不存在，告警
         ./gen_alarm.sh -g "应用进程监控" "thanos rule服务运行状态异常,thanos rule进程不存在,IP:$(hostname -I)" 3 "thanos rule进程" "thanos_rule_pro" ${alarm_file_tmp}
    else
        # 告警存在，恢复原告警
        ./gen_alarm.sh -r "应用进程监控" "thanos rule进程恢复正常,IP:$(hostname -I)" "thanos rule进程" "thanos_rule_pro" ${alarm_file_tmp}

        # 端口检查
        thanos_rule_port_check
    fi
}

# Thanos Rule运行端口监控
thanos_rule_port_check() {
	for port in $(echo ${thanos_rule_port} | sed "s/,/ /g"); do
		port_info=$(netstat -nplt | grep ":${port} ")
		if [[ ${port_info} == '' ]]; then
			# 端口未监听，告警
			./gen_alarm.sh -g "端口状态" "thanos rule服务端口[${thanos_rule_port}]未监听,IP:$(hostname -I)" 3 "thanos rule端口状态" "thanos_rule_port" ${alarm_file_tmp}
		else
			# 端口正常且存在告警，需要恢复告警
			./gen_alarm.sh -r "端口状态" "thanos rule服务端口[${thanos_rule_port}]恢复正常,IP:$(hostname -I)" "thanos rule端口状态" "thanos_rule_port" ${alarm_file_tmp}
		fi
	done
}

while true; do
    holidays_date=$(cat config.cnf | grep Holidays | awk -F "=" '{print $2}' | sed "s/,/ /g")
	current_date=$(date +'%Y-%m-%d')
	echo $holidays_date | grep -o $current_date >> /dev/null
	if [[ $? -eq 0 ]]; then
		exit 0
	else
		echo -n "" >${alarm_file_tmp}
		thanos_rule_pro_check
		# 将缓存告警放入告警文件夹
		./gen_alarm.sh -s ${alarm_file_tmp}
		sleep ${execute_cycle}
	fi
done

